/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package lucene.Analyzers;

import java.io.IOException;
import java.io.Reader;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;

public class PositionTwitterFilter extends TokenFilter{

    private Set twitterTerms;
    
    public  PositionTwitterFilter(TokenStream in) {
        super(in);        
        initSet();
    }
    

    private void initSet(){
        this.twitterTerms = new HashSet<String> (); 
        twitterTerms.add("http");
        twitterTerms.add("RT");
        twitterTerms.add("rt"); 
        //nota: nao ha palavras em ingles que comecem com rt... problemas com nomes proprios?
        twitterTerms.add("@");
        //twitterTerms.add("#"); //vou aceitar hashtags somente para computar similaridade
    }
    
    @Override
    public final Token next() throws IOException {
        int increment = 0;
        boolean isTwitterTerm;
        
        for (Token token = input.next(); token != null; token = input.next()) {
            isTwitterTerm = false;
            Iterator<String> iterator = this.twitterTerms.iterator();
            while (iterator.hasNext() && !isTwitterTerm){
                if (token.termText().startsWith(iterator.next()))
                    isTwitterTerm = true;                    
            }                
            if (!isTwitterTerm){ 
                token.setPositionIncrement(token.getPositionIncrement() + increment);
                return token;
            }
        increment++;
        }
        return null;
    }
}
